From 76974398a63c1da79c4bd1dceb9d6d51d144f037 Mon Sep 17 00:00:00 2001 From: "kaf24@viper.(none)" Date: Sun, 6 Feb 2005 21:30:19 +0000 Subject: [PATCH] bitkeeper revision 1.1159.212.106 (42068c6bV88PUeuTyW0W65OVudMAlQ) Added user-memory accessing functionality for x86_64. Signed-off-by: keir.fraser@cl.cam.ac.uk --- xen/arch/x86/boot/x86_64.S | 9 - xen/arch/x86/x86_32/usercopy.c | 137 -------- xen/arch/x86/x86_64/usercopy.c | 194 +++++++----- xen/common/dom_mem_ops.c | 12 +- xen/include/asm-x86/x86_32/uaccess.h | 124 +++----- xen/include/asm-x86/x86_64/uaccess.h | 449 +++++++++++++-------------- 6 files changed, 382 insertions(+), 543 deletions(-) diff --git a/xen/arch/x86/boot/x86_64.S b/xen/arch/x86/boot/x86_64.S index 1ff32ba47c..319f18a54b 100644 --- a/xen/arch/x86/boot/x86_64.S +++ b/xen/arch/x86/boot/x86_64.S @@ -248,12 +248,3 @@ ENTRY(cpu0_stack) # Initial stack is 8kB .org 0x6000 ENTRY(stext) ENTRY(_stext) - -.globl copy_from_user, copy_to_user, copy_user_generic -copy_from_user: -copy_to_user: -copy_user_generic: -.globl __get_user_1, __get_user_4, __get_user_8 -__get_user_1: -__get_user_4: -__get_user_8: diff --git a/xen/arch/x86/x86_32/usercopy.c b/xen/arch/x86/x86_32/usercopy.c index df30b4849c..7e479bbee8 100644 --- a/xen/arch/x86/x86_32/usercopy.c +++ b/xen/arch/x86/x86_32/usercopy.c @@ -9,8 +9,6 @@ #include #include -#define might_sleep() ((void)0) - static inline int __movsl_is_ok(unsigned long a1, unsigned long a2, unsigned long n) { #ifdef CONFIG_X86_INTEL_USERCOPY @@ -22,93 +20,6 @@ static inline int __movsl_is_ok(unsigned long a1, unsigned long a2, unsigned lon #define movsl_is_ok(a1,a2,n) \ __movsl_is_ok((unsigned long)(a1),(unsigned long)(a2),(n)) -/* - * Copy a null terminated string from userspace. - */ - -#define __do_strncpy_from_user(dst,src,count,res) \ -do { \ - int __d0, __d1, __d2; \ - __asm__ __volatile__( \ - " testl %1,%1\n" \ - " jz 2f\n" \ - "0: lodsb\n" \ - " stosb\n" \ - " testb %%al,%%al\n" \ - " jz 1f\n" \ - " decl %1\n" \ - " jnz 0b\n" \ - "1: subl %1,%0\n" \ - "2:\n" \ - ".section .fixup,\"ax\"\n" \ - "3: movl %5,%0\n" \ - " jmp 2b\n" \ - ".previous\n" \ - ".section __ex_table,\"a\"\n" \ - " .align 4\n" \ - " .long 0b,3b\n" \ - ".previous" \ - : "=d"(res), "=c"(count), "=&a" (__d0), "=&S" (__d1), \ - "=&D" (__d2) \ - : "i"(-EFAULT), "0"(count), "1"(count), "3"(src), "4"(dst) \ - : "memory"); \ -} while (0) - -/** - * __strncpy_from_user: - Copy a NUL terminated string from userspace, with less checking. - * @dst: Destination address, in kernel space. This buffer must be at - * least @count bytes long. - * @src: Source address, in user space. - * @count: Maximum number of bytes to copy, including the trailing NUL. - * - * Copies a NUL-terminated string from userspace to kernel space. - * Caller must check the specified block with access_ok() before calling - * this function. - * - * On success, returns the length of the string (not including the trailing - * NUL). - * - * If access to userspace fails, returns -EFAULT (some data may have been - * copied). - * - * If @count is smaller than the length of the string, copies @count bytes - * and returns @count. - */ -long -__strncpy_from_user(char *dst, const char __user *src, long count) -{ - long res; - __do_strncpy_from_user(dst, src, count, res); - return res; -} - -/** - * strncpy_from_user: - Copy a NUL terminated string from userspace. - * @dst: Destination address, in kernel space. This buffer must be at - * least @count bytes long. - * @src: Source address, in user space. - * @count: Maximum number of bytes to copy, including the trailing NUL. - * - * Copies a NUL-terminated string from userspace to kernel space. - * - * On success, returns the length of the string (not including the trailing - * NUL). - * - * If access to userspace fails, returns -EFAULT (some data may have been - * copied). - * - * If @count is smaller than the length of the string, copies @count bytes - * and returns @count. - */ -long -strncpy_from_user(char *dst, const char __user *src, long count) -{ - long res = -EFAULT; - if (access_ok(VERIFY_READ, src, 1)) - __do_strncpy_from_user(dst, src, count, res); - return res; -} - /* * Zero Userspace @@ -148,7 +59,6 @@ do { \ unsigned long clear_user(void __user *to, unsigned long n) { - might_sleep(); if (access_ok(VERIFY_WRITE, to, n)) __do_clear_user(to, n); return n; @@ -172,49 +82,6 @@ __clear_user(void __user *to, unsigned long n) return n; } -/** - * strlen_user: - Get the size of a string in user space. - * @s: The string to measure. - * @n: The maximum valid length - * - * Get the size of a NUL-terminated string in user space. - * - * Returns the size of the string INCLUDING the terminating NUL. - * On exception, returns 0. - * If the string is too long, returns a value greater than @n. - */ -long strnlen_user(const char __user *s, long n) -{ - unsigned long mask = -__addr_ok(s); - unsigned long res, tmp; - - might_sleep(); - - __asm__ __volatile__( - " testl %0, %0\n" - " jz 3f\n" - " andl %0,%%ecx\n" - "0: repne; scasb\n" - " setne %%al\n" - " subl %%ecx,%0\n" - " addl %0,%%eax\n" - "1:\n" - ".section .fixup,\"ax\"\n" - "2: xorl %%eax,%%eax\n" - " jmp 1b\n" - "3: movb $1,%%al\n" - " jmp 1b\n" - ".previous\n" - ".section __ex_table,\"a\"\n" - " .align 4\n" - " .long 0b,2b\n" - ".previous" - :"=r" (n), "=D" (s), "=a" (res), "=c" (tmp) - :"0" (n), "1" (s), "2" (0), "3" (mask) - :"cc"); - return res & mask; -} - #ifdef CONFIG_X86_INTEL_USERCOPY static unsigned long __copy_user_intel(void __user *to, const void *from, unsigned long size) @@ -543,12 +410,10 @@ __copy_from_user_ll(void *to, const void __user *from, unsigned long n) unsigned long copy_to_user(void __user *to, const void *from, unsigned long n) { - might_sleep(); if (access_ok(VERIFY_WRITE, to, n)) n = __copy_to_user(to, from, n); return n; } -EXPORT_SYMBOL(copy_to_user); /** * copy_from_user: - Copy a block of data from user space. @@ -569,11 +434,9 @@ EXPORT_SYMBOL(copy_to_user); unsigned long copy_from_user(void *to, const void __user *from, unsigned long n) { - might_sleep(); if (access_ok(VERIFY_READ, from, n)) n = __copy_from_user(to, from, n); else memset(to, 0, n); return n; } -EXPORT_SYMBOL(copy_from_user); diff --git a/xen/arch/x86/x86_64/usercopy.c b/xen/arch/x86/x86_64/usercopy.c index c060c45890..839d2582a5 100644 --- a/xen/arch/x86/x86_64/usercopy.c +++ b/xen/arch/x86/x86_64/usercopy.c @@ -7,55 +7,6 @@ */ #include -/* - * Copy a null terminated string from userspace. - */ - -#define __do_strncpy_from_user(dst,src,count,res) \ -do { \ - long __d0, __d1, __d2; \ - __asm__ __volatile__( \ - " testq %1,%1\n" \ - " jz 2f\n" \ - "0: lodsb\n" \ - " stosb\n" \ - " testb %%al,%%al\n" \ - " jz 1f\n" \ - " decq %1\n" \ - " jnz 0b\n" \ - "1: subq %1,%0\n" \ - "2:\n" \ - ".section .fixup,\"ax\"\n" \ - "3: movq %5,%0\n" \ - " jmp 2b\n" \ - ".previous\n" \ - ".section __ex_table,\"a\"\n" \ - " .align 8\n" \ - " .quad 0b,3b\n" \ - ".previous" \ - : "=r"(res), "=c"(count), "=&a" (__d0), "=&S" (__d1), \ - "=&D" (__d2) \ - : "i"(-EFAULT), "0"(count), "1"(count), "3"(src), "4"(dst) \ - : "memory"); \ -} while (0) - -long -__strncpy_from_user(char *dst, const char *src, long count) -{ - long res; - __do_strncpy_from_user(dst, src, count, res); - return res; -} - -long -strncpy_from_user(char *dst, const char *src, long count) -{ - long res = -EFAULT; - if (access_ok(VERIFY_READ, src, 1)) - __do_strncpy_from_user(dst, src, count, res); - return res; -} - /* * Zero Userspace */ @@ -93,6 +44,86 @@ unsigned long __clear_user(void *addr, unsigned long size) return size; } +unsigned long __copy_to_user_ll(void __user *to, const void *from, unsigned n) +{ + unsigned long __d0, __d1, __d2, __n = n; + __asm__ __volatile__( + " cmpq $15,%0\n" + " jbe 1f\n" + " mov %1,%0\n" + " neg %0\n" + " and $7,%0\n" + " sub %0,%3\n" + "4: rep; movsb\n" /* make 'to' address aligned */ + " mov %3,%0\n" + " shr $3,%0\n" + " and $7,%3\n" + " .align 2,0x90\n" + "0: rep; movsq\n" /* as many quadwords as possible... */ + " mov %3,%0\n" + "1: rep; movsb\n" /* ...remainder copied as bytes */ + "2:\n" + ".section .fixup,\"ax\"\n" + "5: add %3,%0\n" + " jmp 2b\n" + "3: lea 0(%3,%0,8),%0\n" + " jmp 2b\n" + ".previous\n" + ".section __ex_table,\"a\"\n" + " .align 8\n" + " .quad 4b,5b\n" + " .quad 0b,3b\n" + " .quad 1b,2b\n" + ".previous" + : "=&c"(__n), "=&D" (__d0), "=&S" (__d1), "=r"(__d2) + : "3"(__n), "0"(__n), "1"(to), "2"(from) + : "memory"); + return (unsigned)__n; +} + +unsigned long +__copy_from_user_ll(void *to, const void __user *from, unsigned n) +{ + unsigned long __d0, __d1, __d2, __n = n; + __asm__ __volatile__( + " cmp $15,%0\n" + " jbe 1f\n" + " mov %1,%0\n" + " neg %0\n" + " and $7,%0\n" + " sub %0,%3\n" + "4: rep; movsb\n" /* make 'to' address aligned */ + " mov %3,%0\n" + " shr $3,%0\n" + " and $7,%3\n" + " .align 2,0x90\n" + "0: rep; movsq\n" /* as many quadwords as possible... */ + " mov %3,%0\n" + "1: rep; movsb\n" /* ...remainder copied as bytes */ + "2:\n" + ".section .fixup,\"ax\"\n" + "5: add %3,%0\n" + " jmp 6f\n" + "3: lea 0(%3,%0,8),%0\n" + "6: push %0\n" + " push %%rax\n" + " xor %%rax,%%rax\n" + " rep; stosb\n" + " pop %%rax\n" + " pop %0\n" + " jmp 2b\n" + ".previous\n" + ".section __ex_table,\"a\"\n" + " .align 8\n" + " .quad 4b,5b\n" + " .quad 0b,3b\n" + " .quad 1b,6b\n" + ".previous" + : "=&c"(__n), "=&D" (__d0), "=&S" (__d1), "=r"(__d2) + : "3"(__n), "0"(__n), "1"(to), "2"(from) + : "memory"); + return (unsigned)__n; +} unsigned long clear_user(void *to, unsigned long n) { @@ -101,36 +132,49 @@ unsigned long clear_user(void *to, unsigned long n) return n; } -/* - * Return the size of a string (including the ending 0) +/** + * copy_to_user: - Copy a block of data into user space. + * @to: Destination address, in user space. + * @from: Source address, in kernel space. + * @n: Number of bytes to copy. + * + * Context: User context only. This function may sleep. + * + * Copy data from kernel space to user space. * - * Return 0 on exception, a value greater than N if too long + * Returns number of bytes that could not be copied. + * On success, this will be zero. */ - -long strnlen_user(const char *s, long n) +unsigned long +copy_to_user(void __user *to, const void *from, unsigned n) { - unsigned long res = 0; - char c; - - if (!access_ok(VERIFY_READ, s, n)) - return 0; - - while (1) { - if (get_user(c, s)) - return 0; - if (!c) - return res+1; - if (res>n) - return n+1; - res++; - s++; - } + if (access_ok(VERIFY_WRITE, to, n)) + n = __copy_to_user(to, from, n); + return n; } -unsigned long copy_in_user(void *to, const void *from, unsigned len) +/** + * copy_from_user: - Copy a block of data from user space. + * @to: Destination address, in kernel space. + * @from: Source address, in user space. + * @n: Number of bytes to copy. + * + * Context: User context only. This function may sleep. + * + * Copy data from user space to kernel space. + * + * Returns number of bytes that could not be copied. + * On success, this will be zero. + * + * If some data could not be copied, this function will pad the copied + * data to the requested size using zero bytes. + */ +unsigned long +copy_from_user(void *to, const void __user *from, unsigned n) { - if (access_ok(VERIFY_WRITE, to, len) && access_ok(VERIFY_READ, from, len)) { - return copy_user_generic(to, from, len); - } - return len; + if (access_ok(VERIFY_READ, from, n)) + n = __copy_from_user(to, from, n); + else + memset(to, 0, n); + return n; } diff --git a/xen/common/dom_mem_ops.c b/xen/common/dom_mem_ops.c index f9fae770fc..9de62004ba 100644 --- a/xen/common/dom_mem_ops.c +++ b/xen/common/dom_mem_ops.c @@ -35,7 +35,7 @@ static long alloc_dom_mem(struct domain *d, unsigned long *extent_list, unsigned long start_extent, - unsigned long nr_extents, + unsigned int nr_extents, unsigned int extent_order) { struct pfn_info *page; @@ -73,7 +73,7 @@ static long free_dom_mem(struct domain *d, unsigned long *extent_list, unsigned long start_extent, - unsigned long nr_extents, + unsigned int nr_extents, unsigned int extent_order) { struct pfn_info *page; @@ -134,7 +134,7 @@ do_dom_mem_op(unsigned long op, op &= (1 << START_EXTENT_SHIFT) - 1; if ( unlikely(start_extent > nr_extents) || - unlikely(nr_extents > (~0UL >> START_EXTENT_SHIFT)) ) + unlikely(nr_extents > ~0U) ) /* can pack into a uint? */ return -EINVAL; if ( likely(domid == DOMID_SELF) ) @@ -150,11 +150,13 @@ do_dom_mem_op(unsigned long op, { case MEMOP_increase_reservation: rc = alloc_dom_mem( - d, extent_list, start_extent, nr_extents, extent_order); + d, extent_list, start_extent, + (unsigned int)nr_extents, extent_order); break; case MEMOP_decrease_reservation: rc = free_dom_mem( - d, extent_list, start_extent, nr_extents, extent_order); + d, extent_list, start_extent, + (unsigned int)nr_extents, extent_order); break; default: rc = -ENOSYS; diff --git a/xen/include/asm-x86/x86_32/uaccess.h b/xen/include/asm-x86/x86_32/uaccess.h index b202a1a12b..c2420e74e5 100644 --- a/xen/include/asm-x86/x86_32/uaccess.h +++ b/xen/include/asm-x86/x86_32/uaccess.h @@ -10,9 +10,7 @@ #include #include -/* No user-pointer checking. */ #define __user -#define __chk_user_ptr(_p) ((void)0) #define VERIFY_READ 0 #define VERIFY_WRITE 1 @@ -22,7 +20,7 @@ */ #ifdef CONFIG_X86_INTEL_USERCOPY extern struct movsl_mask { - int mask; + int mask; } __cacheline_aligned movsl_mask; #endif @@ -34,41 +32,22 @@ extern struct movsl_mask { * * This is equivalent to the following test: * (u33)addr + (u33)size >= (u33)HYPERVISOR_VIRT_START - * - * This needs 33-bit arithmetic. We have a carry... */ -#define __range_ok(addr,size) ({ \ +#define __range_not_ok(addr,size) ({ \ unsigned long flag,sum; \ - __chk_user_ptr(addr); \ asm("addl %3,%1 ; sbbl %0,%0; cmpl %1,%4; sbbl $0,%0" \ :"=&r" (flag), "=r" (sum) \ :"1" (addr),"g" ((int)(size)),"r" (HYPERVISOR_VIRT_START)); \ flag; }) -/** - * access_ok: - Checks if a user space pointer is valid - * @type: Type of access: %VERIFY_READ or %VERIFY_WRITE. Note that - * %VERIFY_WRITE is a superset of %VERIFY_READ - if it is safe - * to write to a block, it is always safe to read from it. - * @addr: User space pointer to start of block to check - * @size: Size of block to check - * - * Context: User context only. This function may sleep. - * - * Checks if a pointer to a block of memory in user space is valid. - * - * Returns true (nonzero) if the memory block may be valid, false (zero) - * if it is definitely invalid. - * - * Note that, depending on architecture, this function probably just - * checks that the pointer is in the user space range - after calling - * this function, memory access functions may still return -EFAULT. - */ -#define access_ok(type,addr,size) (likely(__range_ok(addr,size) == 0)) +#define access_ok(type,addr,size) (likely(__range_not_ok(addr,size) == 0)) #define array_access_ok(type,addr,count,size) \ (likely(count < (~0UL/size)) && access_ok(type,addr,count*size)) +extern long __get_user_bad(void); +extern void __put_user_bad(void); + /** * get_user: - Get a simple variable from user space. * @x: Variable to store result. @@ -89,8 +68,6 @@ extern struct movsl_mask { #define get_user(x,ptr) \ __get_user_check((x),(ptr),sizeof(*(ptr))) -extern void __put_user_bad(void); - /** * put_user: - Write a simple value into user space. * @x: Value to copy to user space. @@ -195,7 +172,6 @@ extern void __put_user_bad(void); #define __put_user_size(x,ptr,size,retval,errret) \ do { \ retval = 0; \ - __chk_user_ptr(ptr); \ switch (size) { \ case 1: __put_user_asm(x,ptr,retval,"b","b","iq",errret);break; \ case 2: __put_user_asm(x,ptr,retval,"w","w","ir",errret);break; \ @@ -259,12 +235,9 @@ struct __large_struct { unsigned long buf[100]; }; __gu_err; \ }) -extern long __get_user_bad(void); - #define __get_user_size(x,ptr,size,retval,errret) \ do { \ retval = 0; \ - __chk_user_ptr(ptr); \ switch (size) { \ case 1: __get_user_asm(x,ptr,retval,"b","b","=q",errret);break; \ case 2: __get_user_asm(x,ptr,retval,"w","w","=r",errret);break; \ @@ -317,22 +290,22 @@ unsigned long __copy_from_user_ll(void *to, const void __user *from, unsigned lo static always_inline unsigned long __copy_to_user(void __user *to, const void *from, unsigned long n) { - if (__builtin_constant_p(n)) { - unsigned long ret; - - switch (n) { - case 1: - __put_user_size(*(u8 *)from, (u8 __user *)to, 1, ret, 1); - return ret; - case 2: - __put_user_size(*(u16 *)from, (u16 __user *)to, 2, ret, 2); - return ret; - case 4: - __put_user_size(*(u32 *)from, (u32 __user *)to, 4, ret, 4); - return ret; - } - } - return __copy_to_user_ll(to, from, n); + if (__builtin_constant_p(n)) { + unsigned long ret; + + switch (n) { + case 1: + __put_user_size(*(u8 *)from, (u8 __user *)to, 1, ret, 1); + return ret; + case 2: + __put_user_size(*(u16 *)from, (u16 __user *)to, 2, ret, 2); + return ret; + case 4: + __put_user_size(*(u32 *)from, (u32 __user *)to, 4, ret, 4); + return ret; + } + } + return __copy_to_user_ll(to, from, n); } /** @@ -355,47 +328,28 @@ __copy_to_user(void __user *to, const void *from, unsigned long n) static always_inline unsigned long __copy_from_user(void *to, const void __user *from, unsigned long n) { - if (__builtin_constant_p(n)) { - unsigned long ret; - - switch (n) { - case 1: - __get_user_size(*(u8 *)to, from, 1, ret, 1); - return ret; - case 2: - __get_user_size(*(u16 *)to, from, 2, ret, 2); - return ret; - case 4: - __get_user_size(*(u32 *)to, from, 4, ret, 4); - return ret; - } - } - return __copy_from_user_ll(to, from, n); + if (__builtin_constant_p(n)) { + unsigned long ret; + + switch (n) { + case 1: + __get_user_size(*(u8 *)to, from, 1, ret, 1); + return ret; + case 2: + __get_user_size(*(u16 *)to, from, 2, ret, 2); + return ret; + case 4: + __get_user_size(*(u32 *)to, from, 4, ret, 4); + return ret; + } + } + return __copy_from_user_ll(to, from, n); } unsigned long copy_to_user(void __user *to, const void *from, unsigned long n); unsigned long copy_from_user(void *to, - const void __user *from, unsigned long n); -long strncpy_from_user(char *dst, const char __user *src, long count); -long __strncpy_from_user(char *dst, const char __user *src, long count); - -/** - * strlen_user: - Get the size of a string in user space. - * @str: The string to measure. - * - * Context: User context only. This function may sleep. - * - * Get the size of a NUL-terminated string in user space. - * - * Returns the size of the string INCLUDING the terminating NUL. - * On exception, returns 0. - * - * If there is a limit on the length of a valid string, you may wish to - * consider using strnlen_user() instead. - */ -#define strlen_user(str) strnlen_user(str, ~0UL >> 1) + const void __user *from, unsigned long n); -long strnlen_user(const char __user *str, long n); unsigned long clear_user(void __user *mem, unsigned long len); unsigned long __clear_user(void __user *mem, unsigned long len); diff --git a/xen/include/asm-x86/x86_64/uaccess.h b/xen/include/asm-x86/x86_64/uaccess.h index f965c87d32..52789d401e 100644 --- a/xen/include/asm-x86/x86_64/uaccess.h +++ b/xen/include/asm-x86/x86_64/uaccess.h @@ -11,10 +11,7 @@ #include #include -/* No user-pointer checking. */ #define __user -#define __force -#define __chk_user_ptr(_p) ((void)0) #define VERIFY_READ 0 #define VERIFY_WRITE 1 @@ -22,122 +19,149 @@ #define __addr_ok(addr) ((unsigned long)(addr) < HYPERVISOR_VIRT_START) /* - * Uhhuh, this needs 65-bit arithmetic. We have a carry.. + * Test whether a block of memory is a valid user space address. + * Returns 0 if the range is valid, nonzero otherwise. + * + * This is equivalent to the following test: + * ((u65)addr >= (u65)HYPERVISOR_VIRT_END) ? + * (((u65)addr + (u65)size) >= ((u65)1 << 64)) : + * (((u65)addr + (u65)size) >= ((u65)HYPERVISOR_VIRT_START)) */ #define __range_not_ok(addr,size) ({ \ - unsigned long flag,sum; \ - __chk_user_ptr(addr); \ - asm("# range_ok\n\r" \ - "addq %3,%1 ; sbbq %0,%0 ; cmpq %1,%4 ; sbbq $0,%0" \ - :"=&r" (flag), "=r" (sum) \ - :"1" (addr),"g" ((long)(size)),"r" (HYPERVISOR_VIRT_START)); \ - flag; }) + unsigned long flag,sum; \ + if ((unsigned long)addr >= HYPERVISOR_VIRT_END) \ + asm("addq %3,%1 ; sbbq %0,%0" \ + :"=&r" (flag), "=r" (sum) \ + :"1" (addr),"g" ((long)(size))); \ + else \ + asm("addq %3,%1 ; sbbq %0,%0 ; cmpq %1,%4 ; sbbq $0,%0" \ + :"=&r" (flag), "=r" (sum) \ + :"1" (addr),"g" ((long)(size)),"r" (HYPERVISOR_VIRT_START)); \ + flag; }) #define access_ok(type, addr, size) (__range_not_ok(addr,size) == 0) #define array_access_ok(type,addr,count,size) \ (likely(sizeof(count) <= 4) /* disallow 64-bit counts */ && \ - access_ok(type,addr,count*size)) + access_ok(type,addr,(unsigned long)count*(unsigned long)size)) -extern inline int verify_area(int type, const void __user * addr, unsigned long size) -{ - return access_ok(type,addr,size) ? 0 : -EFAULT; -} +extern long __get_user_bad(void); +extern void __put_user_bad(void); -/* - * These are the main single-value transfer routines. They automatically - * use the right size if we just have the right pointer type. +/** + * get_user: - Get a simple variable from user space. + * @x: Variable to store result. + * @ptr: Source address, in user space. * - * This gets kind of ugly. We want to return _two_ values in "get_user()" - * and yet we don't want to do any pointers, because that is too much - * of a performance impact. Thus we have a few rather ugly macros here, - * and hide all the ugliness from the user. + * Context: User context only. This function may sleep. * - * The "__xxx" versions of the user access functions are versions that - * do not verify the address space, that must have been done previously - * with a separate "access_ok()" call (this is used when we do multiple - * accesses to the same area of user memory). + * This macro copies a single simple variable from user space to kernel + * space. It supports simple types like char and int, but not larger + * data types like structures or arrays. + * + * @ptr must have pointer-to-simple-variable type, and the result of + * dereferencing @ptr must be assignable to @x without a cast. + * + * Returns zero on success, or -EFAULT on error. + * On error, the variable @x is set to zero. */ +#define get_user(x,ptr) \ + __get_user_check((x),(ptr),sizeof(*(ptr))) -extern void __get_user_1(void); -extern void __get_user_2(void); -extern void __get_user_4(void); -extern void __get_user_8(void); - -#define __get_user_x(size,ret,x,ptr) \ - __asm__ __volatile__("call __get_user_" #size \ - :"=a" (ret),"=d" (x) \ - :"0" (ptr) \ - :"rbx") - -/* Careful: we have to cast the result to the type of the pointer for sign reasons */ -#define get_user(x,ptr) \ -({ long __val_gu; \ - int __ret_gu; \ - __chk_user_ptr(ptr); \ - switch(sizeof (*(ptr))) { \ - case 1: __get_user_x(1,__ret_gu,__val_gu,ptr); break; \ - case 2: __get_user_x(2,__ret_gu,__val_gu,ptr); break; \ - case 4: __get_user_x(4,__ret_gu,__val_gu,ptr); break; \ - case 8: __get_user_x(8,__ret_gu,__val_gu,ptr); break; \ - default: __get_user_bad(); break; \ - } \ - (x) = (__typeof__(*(ptr)))__val_gu; \ - __ret_gu; \ -}) - -extern void __put_user_1(void); -extern void __put_user_2(void); -extern void __put_user_4(void); -extern void __put_user_8(void); - -extern void __put_user_bad(void); - -#define __put_user_x(size,ret,x,ptr) \ - __asm__ __volatile__("call __put_user_" #size \ - :"=a" (ret) \ - :"0" (ptr),"d" (x) \ - :"rbx") - +/** + * put_user: - Write a simple value into user space. + * @x: Value to copy to user space. + * @ptr: Destination address, in user space. + * + * Context: User context only. This function may sleep. + * + * This macro copies a single simple value from kernel space to user + * space. It supports simple types like char and int, but not larger + * data types like structures or arrays. + * + * @ptr must have pointer-to-simple-variable type, and @x must be assignable + * to the result of dereferencing @ptr. + * + * Returns zero on success, or -EFAULT on error. + */ #define put_user(x,ptr) \ __put_user_check((__typeof__(*(ptr)))(x),(ptr),sizeof(*(ptr))) + +/** + * __get_user: - Get a simple variable from user space, with less checking. + * @x: Variable to store result. + * @ptr: Source address, in user space. + * + * Context: User context only. This function may sleep. + * + * This macro copies a single simple variable from user space to kernel + * space. It supports simple types like char and int, but not larger + * data types like structures or arrays. + * + * @ptr must have pointer-to-simple-variable type, and the result of + * dereferencing @ptr must be assignable to @x without a cast. + * + * Caller must check the pointer with access_ok() before calling this + * function. + * + * Returns zero on success, or -EFAULT on error. + * On error, the variable @x is set to zero. + */ #define __get_user(x,ptr) \ __get_user_nocheck((x),(ptr),sizeof(*(ptr))) + + +/** + * __put_user: - Write a simple value into user space, with less checking. + * @x: Value to copy to user space. + * @ptr: Destination address, in user space. + * + * Context: User context only. This function may sleep. + * + * This macro copies a single simple value from kernel space to user + * space. It supports simple types like char and int, but not larger + * data types like structures or arrays. + * + * @ptr must have pointer-to-simple-variable type, and @x must be assignable + * to the result of dereferencing @ptr. + * + * Caller must check the pointer with access_ok() before calling this + * function. + * + * Returns zero on success, or -EFAULT on error. + */ #define __put_user(x,ptr) \ __put_user_nocheck((__typeof__(*(ptr)))(x),(ptr),sizeof(*(ptr))) -#define __put_user_nocheck(x,ptr,size) \ -({ \ - int __pu_err; \ - __put_user_size((x),(ptr),(size),__pu_err); \ - __pu_err; \ +#define __put_user_nocheck(x,ptr,size) \ +({ \ + long __pu_err; \ + __put_user_size((x),(ptr),(size),__pu_err,-EFAULT); \ + __pu_err; \ }) +#define __put_user_check(x,ptr,size) \ +({ \ + long __pu_err = -EFAULT; \ + __typeof__(*(ptr)) __user *__pu_addr = (ptr); \ + if (__addr_ok(__pu_addr)) \ + __put_user_size((x),__pu_addr,(size),__pu_err,-EFAULT); \ + __pu_err; \ +}) -#define __put_user_check(x,ptr,size) \ -({ \ - int __pu_err = -EFAULT; \ - __typeof__(*(ptr)) __user *__pu_addr = (ptr); \ - if (likely(access_ok(VERIFY_WRITE,__pu_addr,size))) \ - __put_user_size((x),__pu_addr,(size),__pu_err); \ - __pu_err; \ -}) - -#define __put_user_size(x,ptr,size,retval) \ +#define __put_user_size(x,ptr,size,retval,errret) \ do { \ retval = 0; \ - __chk_user_ptr(ptr); \ switch (size) { \ - case 1: __put_user_asm(x,ptr,retval,"b","b","iq",-EFAULT); break;\ - case 2: __put_user_asm(x,ptr,retval,"w","w","ir",-EFAULT); break;\ - case 4: __put_user_asm(x,ptr,retval,"l","k","ir",-EFAULT); break;\ - case 8: __put_user_asm(x,ptr,retval,"q","","ir",-EFAULT); break;\ - default: __put_user_bad(); \ + case 1: __put_user_asm(x,ptr,retval,"b","b","iq",errret);break; \ + case 2: __put_user_asm(x,ptr,retval,"w","w","ir",errret);break; \ + case 4: __put_user_asm(x,ptr,retval,"l","k","ir",errret);break; \ + case 8: __put_user_asm(x,ptr,retval,"q","","ir",errret);break; \ + default: __put_user_bad(); \ } \ } while (0) -/* FIXME: this hack is definitely wrong -AK */ struct __large_struct { unsigned long buf[100]; }; #define __m(x) (*(struct __large_struct *)(x)) @@ -146,178 +170,139 @@ struct __large_struct { unsigned long buf[100]; }; * we do not write to any memory gcc knows about, so there are no * aliasing issues. */ -#define __put_user_asm(x, addr, err, itype, rtype, ltype, errno) \ - __asm__ __volatile__( \ - "1: mov"itype" %"rtype"1,%2\n" \ - "2:\n" \ - ".section .fixup,\"ax\"\n" \ - "3: mov %3,%0\n" \ - " jmp 2b\n" \ - ".previous\n" \ - ".section __ex_table,\"a\"\n" \ - " .align 8\n" \ - " .quad 1b,3b\n" \ - ".previous" \ - : "=r"(err) \ - : ltype (x), "m"(__m(addr)), "i"(errno), "0"(err)) - +#define __put_user_asm(x, addr, err, itype, rtype, ltype, errret) \ + __asm__ __volatile__( \ + "1: mov"itype" %"rtype"1,%2\n" \ + "2:\n" \ + ".section .fixup,\"ax\"\n" \ + "3: mov %3,%0\n" \ + " jmp 2b\n" \ + ".previous\n" \ + ".section __ex_table,\"a\"\n" \ + " .align 8\n" \ + " .quad 1b,3b\n" \ + ".previous" \ + : "=r"(err) \ + : ltype (x), "m"(__m(addr)), "i"(errret), "0"(err)) #define __get_user_nocheck(x,ptr,size) \ ({ \ - int __gu_err; \ - long __gu_val; \ - __get_user_size(__gu_val,(ptr),(size),__gu_err); \ + long __gu_err, __gu_val; \ + __get_user_size(__gu_val,(ptr),(size),__gu_err,-EFAULT);\ (x) = (__typeof__(*(ptr)))__gu_val; \ __gu_err; \ }) -extern int __get_user_bad(void); - -#define __get_user_size(x,ptr,size,retval) \ +#define __get_user_check(x,ptr,size) \ +({ \ + long __gu_err, __gu_val; \ + __typeof__(*(ptr)) __user *__gu_addr = (ptr); \ + __get_user_size(__gu_val,__gu_addr,(size),__gu_err,-EFAULT); \ + (x) = (__typeof__(*(ptr)))__gu_val; \ + if (!__addr_ok(__gu_addr)) __gu_err = -EFAULT; \ + __gu_err; \ +}) + +#define __get_user_size(x,ptr,size,retval,errret) \ do { \ retval = 0; \ - __chk_user_ptr(ptr); \ switch (size) { \ - case 1: __get_user_asm(x,ptr,retval,"b","b","=q",-EFAULT); break;\ - case 2: __get_user_asm(x,ptr,retval,"w","w","=r",-EFAULT); break;\ - case 4: __get_user_asm(x,ptr,retval,"l","k","=r",-EFAULT); break;\ - case 8: __get_user_asm(x,ptr,retval,"q","","=r",-EFAULT); break;\ - default: (x) = __get_user_bad(); \ + case 1: __get_user_asm(x,ptr,retval,"b","b","=q",errret);break; \ + case 2: __get_user_asm(x,ptr,retval,"w","w","=r",errret);break; \ + case 4: __get_user_asm(x,ptr,retval,"l","k","=r",errret);break; \ + case 8: __get_user_asm(x,ptr,retval,"q","","=r",errret); break; \ + default: (x) = __get_user_bad(); \ } \ } while (0) -#define __get_user_asm(x, addr, err, itype, rtype, ltype, errno) \ - __asm__ __volatile__( \ - "1: mov"itype" %2,%"rtype"1\n" \ - "2:\n" \ - ".section .fixup,\"ax\"\n" \ - "3: mov %3,%0\n" \ - " xor"itype" %"rtype"1,%"rtype"1\n" \ - " jmp 2b\n" \ - ".previous\n" \ - ".section __ex_table,\"a\"\n" \ - " .align 8\n" \ - " .quad 1b,3b\n" \ - ".previous" \ - : "=r"(err), ltype (x) \ - : "m"(__m(addr)), "i"(errno), "0"(err)) +#define __get_user_asm(x, addr, err, itype, rtype, ltype, errret) \ + __asm__ __volatile__( \ + "1: mov"itype" %2,%"rtype"1\n" \ + "2:\n" \ + ".section .fixup,\"ax\"\n" \ + "3: mov %3,%0\n" \ + " xor"itype" %"rtype"1,%"rtype"1\n" \ + " jmp 2b\n" \ + ".previous\n" \ + ".section __ex_table,\"a\"\n" \ + " .align 8\n" \ + " .quad 1b,3b\n" \ + ".previous" \ + : "=r"(err), ltype (x) \ + : "m"(__m(addr)), "i"(errret), "0"(err)) + /* * Copy To/From Userspace */ /* Handles exceptions in both to and from, but doesn't do access_ok */ -extern unsigned long copy_user_generic(void *to, const void *from, unsigned len); +unsigned long __copy_to_user_ll(void __user *to, const void *from, unsigned n); +unsigned long __copy_from_user_ll(void *to, const void __user *from, unsigned n); -extern unsigned long copy_to_user(void __user *to, const void *from, unsigned len); -extern unsigned long copy_from_user(void *to, const void __user *from, unsigned len); -extern unsigned long copy_in_user(void __user *to, const void __user *from, unsigned len); +unsigned long copy_to_user(void __user *to, const void *from, unsigned len); +unsigned long copy_from_user(void *to, const void __user *from, unsigned len); static always_inline int __copy_from_user(void *dst, const void __user *src, unsigned size) { - int ret = 0; - if (!__builtin_constant_p(size)) - return copy_user_generic(dst,(__force void *)src,size); - switch (size) { - case 1:__get_user_asm(*(u8*)dst,(u8 __user *)src,ret,"b","b","=q",1); - return ret; - case 2:__get_user_asm(*(u16*)dst,(u16 __user *)src,ret,"w","w","=r",2); - return ret; - case 4:__get_user_asm(*(u32*)dst,(u32 __user *)src,ret,"l","k","=r",4); - return ret; - case 8:__get_user_asm(*(u64*)dst,(u64 __user *)src,ret,"q","","=r",8); - return ret; - case 10: - __get_user_asm(*(u64*)dst,(u64 __user *)src,ret,"q","","=r",16); - if (unlikely(ret)) return ret; - __get_user_asm(*(u16*)(8+(char*)dst),(u16 __user *)(8+(char __user *)src),ret,"w","w","=r",2); - return ret; - case 16: - __get_user_asm(*(u64*)dst,(u64 __user *)src,ret,"q","","=r",16); - if (unlikely(ret)) return ret; - __get_user_asm(*(u64*)(8+(char*)dst),(u64 __user *)(8+(char __user *)src),ret,"q","","=r",8); - return ret; - default: - return copy_user_generic(dst,(__force void *)src,size); - } + int ret = 0; + if (!__builtin_constant_p(size)) + return __copy_from_user_ll(dst,(void *)src,size); + switch (size) { + case 1:__get_user_asm(*(u8*)dst,(u8 __user *)src,ret,"b","b","=q",1); + return ret; + case 2:__get_user_asm(*(u16*)dst,(u16 __user *)src,ret,"w","w","=r",2); + return ret; + case 4:__get_user_asm(*(u32*)dst,(u32 __user *)src,ret,"l","k","=r",4); + return ret; + case 8:__get_user_asm(*(u64*)dst,(u64 __user *)src,ret,"q","","=r",8); + return ret; + case 10: + __get_user_asm(*(u64*)dst,(u64 __user *)src,ret,"q","","=r",16); + if (unlikely(ret)) return ret; + __get_user_asm(*(u16*)(8+(char*)dst),(u16 __user *)(8+(char __user *)src),ret,"w","w","=r",2); + return ret; + case 16: + __get_user_asm(*(u64*)dst,(u64 __user *)src,ret,"q","","=r",16); + if (unlikely(ret)) return ret; + __get_user_asm(*(u64*)(8+(char*)dst),(u64 __user *)(8+(char __user *)src),ret,"q","","=r",8); + return ret; + default: + return __copy_from_user_ll(dst,(void *)src,size); + } } static always_inline int __copy_to_user(void __user *dst, const void *src, unsigned size) { - int ret = 0; - if (!__builtin_constant_p(size)) - return copy_user_generic((__force void *)dst,src,size); - switch (size) { - case 1:__put_user_asm(*(u8*)src,(u8 __user *)dst,ret,"b","b","iq",1); - return ret; - case 2:__put_user_asm(*(u16*)src,(u16 __user *)dst,ret,"w","w","ir",2); - return ret; - case 4:__put_user_asm(*(u32*)src,(u32 __user *)dst,ret,"l","k","ir",4); - return ret; - case 8:__put_user_asm(*(u64*)src,(u64 __user *)dst,ret,"q","","ir",8); - return ret; - case 10: - __put_user_asm(*(u64*)src,(u64 __user *)dst,ret,"q","","ir",10); - if (unlikely(ret)) return ret; - asm("":::"memory"); - __put_user_asm(4[(u16*)src],4+(u16 __user *)dst,ret,"w","w","ir",2); - return ret; - case 16: - __put_user_asm(*(u64*)src,(u64 __user *)dst,ret,"q","","ir",16); - if (unlikely(ret)) return ret; - asm("":::"memory"); - __put_user_asm(1[(u64*)src],1+(u64 __user *)dst,ret,"q","","ir",8); - return ret; - default: - return copy_user_generic((__force void *)dst,src,size); - } -} - - -static always_inline int __copy_in_user(void __user *dst, const void __user *src, unsigned size) -{ - int ret = 0; - if (!__builtin_constant_p(size)) - return copy_user_generic((__force void *)dst,(__force void *)src,size); - switch (size) { - case 1: { - u8 tmp; - __get_user_asm(tmp,(u8 __user *)src,ret,"b","b","=q",1); - if (likely(!ret)) - __put_user_asm(tmp,(u8 __user *)dst,ret,"b","b","iq",1); - return ret; - } - case 2: { - u16 tmp; - __get_user_asm(tmp,(u16 __user *)src,ret,"w","w","=r",2); - if (likely(!ret)) - __put_user_asm(tmp,(u16 __user *)dst,ret,"w","w","ir",2); - return ret; - } - - case 4: { - u32 tmp; - __get_user_asm(tmp,(u32 __user *)src,ret,"l","k","=r",4); - if (likely(!ret)) - __put_user_asm(tmp,(u32 __user *)dst,ret,"l","k","ir",4); - return ret; - } - case 8: { - u64 tmp; - __get_user_asm(tmp,(u64 __user *)src,ret,"q","","=r",8); - if (likely(!ret)) - __put_user_asm(tmp,(u64 __user *)dst,ret,"q","","ir",8); - return ret; - } - default: - return copy_user_generic((__force void *)dst,(__force void *)src,size); - } + int ret = 0; + if (!__builtin_constant_p(size)) + return __copy_to_user_ll((void *)dst,src,size); + switch (size) { + case 1:__put_user_asm(*(u8*)src,(u8 __user *)dst,ret,"b","b","iq",1); + return ret; + case 2:__put_user_asm(*(u16*)src,(u16 __user *)dst,ret,"w","w","ir",2); + return ret; + case 4:__put_user_asm(*(u32*)src,(u32 __user *)dst,ret,"l","k","ir",4); + return ret; + case 8:__put_user_asm(*(u64*)src,(u64 __user *)dst,ret,"q","","ir",8); + return ret; + case 10: + __put_user_asm(*(u64*)src,(u64 __user *)dst,ret,"q","","ir",10); + if (unlikely(ret)) return ret; + asm("":::"memory"); + __put_user_asm(4[(u16*)src],4+(u16 __user *)dst,ret,"w","w","ir",2); + return ret; + case 16: + __put_user_asm(*(u64*)src,(u64 __user *)dst,ret,"q","","ir",16); + if (unlikely(ret)) return ret; + asm("":::"memory"); + __put_user_asm(1[(u64*)src],1+(u64 __user *)dst,ret,"q","","ir",8); + return ret; + default: + return __copy_to_user_ll((void *)dst,src,size); + } } -long strncpy_from_user(char *dst, const char __user *src, long count); -long __strncpy_from_user(char *dst, const char __user *src, long count); -long strnlen_user(const char __user *str, long n); -long strlen_user(const char __user *str); unsigned long clear_user(void __user *mem, unsigned long len); unsigned long __clear_user(void __user *mem, unsigned long len); -- 2.30.2